# -*- coding: utf-8 -*-
# @Time    : 2024/04/01 09:18
# @Author  : Mr.su
# @FileName: __init__.py.py
# @FileDesc:
import pymongo, json
from CollectSpiders.settings import MONGODB_CONFIG


class CollectspidersPipeline:
    myclient = pymongo.MongoClient(MONGODB_CONFIG)
    mydb = myclient["wow_spider_db"]
    db = mydb["tb_product"]
    crawl_count = 0

    def open_spider(self, spider):
        count = 0

    def process_item(self, item, spider):
        # self.db.save(item)
        self.db.update_one({'_id': item['_id']}, {'$setOnInsert': item}, upsert=True)
        self.crawl_count += 1
        return item

    def close_spider(self, spider):
        self.myclient.close()
